{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.4"
    },
    "colab": {
      "name": "LDA.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QefojgwbKiXO",
        "colab_type": "text"
      },
      "source": [
        "# LDA, lanent Dirichlet allocation\n",
        "## 潜在狄利克雷分配"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GH9avKQ1N5E3",
        "colab_type": "text"
      },
      "source": [
        "https://github.com/susanli2016/NLP-with-Python/blob/master/LDA_news_headlines.ipynb"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x6jsXppPKJzP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "\n",
        "data = pd.read_csv('abcnews-date-text.csv', error_bad_lines=False);\n",
        "data_text = data[['headline_text']]\n",
        "data_text['index'] = data_text.index\n",
        "documents = data_text\n",
        "\n",
        "# dataset: https://www.kaggle.com/therohk/million-headlines/data"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qhkr2fWtKJzT",
        "colab_type": "code",
        "outputId": "ff6d388f-be3c-415b-9d63-173f47f63074",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "len(documents)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1103663"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dy7KfZAgKJzX",
        "colab_type": "code",
        "outputId": "dbdd557c-6ec8-4f22-cdae-5cc308c3c79e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "documents[:5]"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>headline_text</th>\n",
              "      <th>index</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>aba decides against community broadcasting lic...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>act fire witnesses must be aware of defamation</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>a g calls for infrastructure protection summit</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>air nz staff in aust strike for pay rise</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>air nz strike to affect australian travellers</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                       headline_text  index\n",
              "0  aba decides against community broadcasting lic...      0\n",
              "1     act fire witnesses must be aware of defamation      1\n",
              "2     a g calls for infrastructure protection summit      2\n",
              "3           air nz staff in aust strike for pay rise      3\n",
              "4      air nz strike to affect australian travellers      4"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JuLzqWIkKJzb",
        "colab_type": "text"
      },
      "source": [
        "### Data Preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-in0sXLLKJzc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import gensim\n",
        "from gensim.utils import simple_preprocess\n",
        "from gensim.parsing.preprocessing import STOPWORDS\n",
        "from nltk.stem import WordNetLemmatizer, SnowballStemmer\n",
        "from nltk.stem.porter import *\n",
        "import numpy as np\n",
        "np.random.seed(2018)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fa38Dq4NKJze",
        "colab_type": "code",
        "outputId": "023a1c63-3553-4edf-cdeb-e2b57d37448d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        }
      },
      "source": [
        "import nltk\n",
        "nltk.download('wordnet')"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Package wordnet is already up-to-date!\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eW1RA3SGKJzg",
        "colab_type": "text"
      },
      "source": [
        "#### Lemmatize example"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ktowKd_QKJzh",
        "colab_type": "code",
        "outputId": "1b2bd3eb-8325-46d7-b262-a4f471603d0d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "print(WordNetLemmatizer().lemmatize('went', pos='v'))"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "go\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XF26OqQAKJzj",
        "colab_type": "text"
      },
      "source": [
        "#### Stemmer Example"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1U2FfQe-KJzk",
        "colab_type": "code",
        "outputId": "e0099ca4-12b1-4057-ac17-82e77a2a776b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 638
        }
      },
      "source": [
        "stemmer = SnowballStemmer('english')\n",
        "original_words = ['caresses', 'flies', 'dies', 'mules', 'denied','died', 'agreed', 'owned', \n",
        "           'humbled', 'sized','meeting', 'stating', 'siezing', 'itemization','sensational', \n",
        "           'traditional', 'reference', 'colonizer','plotted']\n",
        "singles = [stemmer.stem(plural) for plural in original_words]\n",
        "pd.DataFrame(data = {'original word': original_words, 'stemmed': singles})"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>original word</th>\n",
              "      <th>stemmed</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>caresses</td>\n",
              "      <td>caress</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>flies</td>\n",
              "      <td>fli</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>dies</td>\n",
              "      <td>die</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>mules</td>\n",
              "      <td>mule</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>denied</td>\n",
              "      <td>deni</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>died</td>\n",
              "      <td>die</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>agreed</td>\n",
              "      <td>agre</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>owned</td>\n",
              "      <td>own</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>humbled</td>\n",
              "      <td>humbl</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>sized</td>\n",
              "      <td>size</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>meeting</td>\n",
              "      <td>meet</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>stating</td>\n",
              "      <td>state</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>siezing</td>\n",
              "      <td>siez</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>itemization</td>\n",
              "      <td>item</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>sensational</td>\n",
              "      <td>sensat</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>traditional</td>\n",
              "      <td>tradit</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>reference</td>\n",
              "      <td>refer</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>colonizer</td>\n",
              "      <td>colon</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>plotted</td>\n",
              "      <td>plot</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   original word stemmed\n",
              "0       caresses  caress\n",
              "1          flies     fli\n",
              "2           dies     die\n",
              "3          mules    mule\n",
              "4         denied    deni\n",
              "5           died     die\n",
              "6         agreed    agre\n",
              "7          owned     own\n",
              "8        humbled   humbl\n",
              "9          sized    size\n",
              "10       meeting    meet\n",
              "11       stating   state\n",
              "12       siezing    siez\n",
              "13   itemization    item\n",
              "14   sensational  sensat\n",
              "15   traditional  tradit\n",
              "16     reference   refer\n",
              "17     colonizer   colon\n",
              "18       plotted    plot"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qh42eq30KJzn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def lemmatize_stemming(text):\n",
        "    return stemmer.stem(WordNetLemmatizer().lemmatize(text, pos='v'))\n",
        "\n",
        "def preprocess(text):\n",
        "    result = []\n",
        "    for token in gensim.utils.simple_preprocess(text):\n",
        "        if token not in gensim.parsing.preprocessing.STOPWORDS and len(token) > 3:\n",
        "            result.append(lemmatize_stemming(token))\n",
        "    return result"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iyrrRrcQKJzp",
        "colab_type": "code",
        "outputId": "417db109-7dc6-46c7-8974-c5b2d0fb653b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 125
        }
      },
      "source": [
        "doc_sample = documents[documents['index'] == 4310].values[0][0]\n",
        "\n",
        "print('original document: ')\n",
        "words = []\n",
        "for word in doc_sample.split(' '):\n",
        "    words.append(word)\n",
        "print(words)\n",
        "print('\\n\\n tokenized and lemmatized document: ')\n",
        "print(preprocess(doc_sample))"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "original document: \n",
            "['rain', 'helps', 'dampen', 'bushfires']\n",
            "\n",
            "\n",
            " tokenized and lemmatized document: \n",
            "['rain', 'help', 'dampen', 'bushfir']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C8ZCsWKyKJzr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "processed_docs = documents['headline_text'].map(preprocess)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uWfeG_SDKJzt",
        "colab_type": "code",
        "outputId": "359dd71c-992a-4800-8084-38e967348ac2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 215
        }
      },
      "source": [
        "processed_docs[:10]"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0            [decid, communiti, broadcast, licenc]\n",
              "1                               [wit, awar, defam]\n",
              "2           [call, infrastructur, protect, summit]\n",
              "3                      [staff, aust, strike, rise]\n",
              "4             [strike, affect, australian, travel]\n",
              "5               [ambiti, olsson, win, tripl, jump]\n",
              "6           [antic, delight, record, break, barca]\n",
              "7    [aussi, qualifi, stosur, wast, memphi, match]\n",
              "8            [aust, address, secur, council, iraq]\n",
              "9                         [australia, lock, timet]\n",
              "Name: headline_text, dtype: object"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jGBBjFBDKJzx",
        "colab_type": "text"
      },
      "source": [
        "### Bag of words on the dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rc_iRZeDKJzx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "dictionary = gensim.corpora.Dictionary(processed_docs)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RFtaHAq5KJz0",
        "colab_type": "code",
        "outputId": "d37e611a-0f71-4ceb-fdfa-62a09500e8b6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 215
        }
      },
      "source": [
        "count = 0\n",
        "for k, v in dictionary.iteritems():\n",
        "    print(k, v)\n",
        "    count += 1\n",
        "    if count > 10:\n",
        "        break"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0 broadcast\n",
            "1 communiti\n",
            "2 decid\n",
            "3 licenc\n",
            "4 awar\n",
            "5 defam\n",
            "6 wit\n",
            "7 call\n",
            "8 infrastructur\n",
            "9 protect\n",
            "10 summit\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U49jOYQaKJz2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "dictionary.filter_extremes(no_below=15, no_above=0.5, keep_n=100000)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ioYpK4C9KJz4",
        "colab_type": "code",
        "outputId": "61630f5e-d80d-4bf7-90a1-76b2767d476d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "bow_corpus = [dictionary.doc2bow(doc) for doc in processed_docs]\n",
        "bow_corpus[4310]"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[(76, 1), (112, 1), (484, 1), (4015, 1)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aMkhJ-6gKJz7",
        "colab_type": "code",
        "outputId": "dec829f0-4fb7-413a-8d4a-2c9f9a3ece78",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 89
        }
      },
      "source": [
        "bow_doc_4310 = bow_corpus[4310]\n",
        "\n",
        "for i in range(len(bow_doc_4310)):\n",
        "    print(\"Word {} (\\\"{}\\\") appears {} time.\".format(bow_doc_4310[i][0], \n",
        "                                                     dictionary[bow_doc_4310[i][0]], \n",
        "                                                     bow_doc_4310[i][1]))"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Word 76 (\"bushfir\") appears 1 time.\n",
            "Word 112 (\"help\") appears 1 time.\n",
            "Word 484 (\"rain\") appears 1 time.\n",
            "Word 4015 (\"dampen\") appears 1 time.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wROVxefVKJz8",
        "colab_type": "text"
      },
      "source": [
        "### TF-IDF"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NWxsuXTOKJz9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from gensim import corpora, models\n",
        "\n",
        "tfidf = models.TfidfModel(bow_corpus)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pIRPQeCJKJz_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "corpus_tfidf = tfidf[bow_corpus]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WjlaePQ5KJ0B",
        "colab_type": "code",
        "outputId": "d7602b77-1ab9-44f1-938e-e0fc7bd2bd1a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 89
        }
      },
      "source": [
        "from pprint import pprint\n",
        "\n",
        "for doc in corpus_tfidf:\n",
        "    pprint(doc)\n",
        "    break"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[(0, 0.5893154583024485),\n",
            " (1, 0.3892866165028569),\n",
            " (2, 0.49651921997736453),\n",
            " (3, 0.5046106271280878)]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kzxzv7cHKJ0F",
        "colab_type": "text"
      },
      "source": [
        "### Running LDA using Bag of Words"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lyFUMwotKJ0G",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "lda_model = gensim.models.LdaMulticore(bow_corpus, num_topics=10, id2word=dictionary, passes=2, workers=2)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TF_5iVw8KJ0J",
        "colab_type": "code",
        "outputId": "524af525-0eff-4f3a-b3b5-370e5ab5dc66",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 377
        }
      },
      "source": [
        "for idx, topic in lda_model.print_topics(-1):\n",
        "    print('Topic: {} \\nWords: {}'.format(idx, topic))"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Topic: 0 \n",
            "Words: 0.023*\"death\" + 0.020*\"hospit\" + 0.016*\"trial\" + 0.014*\"driver\" + 0.012*\"street\" + 0.011*\"flood\" + 0.010*\"road\" + 0.010*\"babi\" + 0.010*\"east\" + 0.009*\"wall\"\n",
            "Topic: 1 \n",
            "Words: 0.025*\"world\" + 0.019*\"test\" + 0.016*\"interview\" + 0.014*\"australia\" + 0.012*\"leagu\" + 0.010*\"close\" + 0.010*\"darwin\" + 0.009*\"take\" + 0.009*\"deal\" + 0.009*\"star\"\n",
            "Topic: 2 \n",
            "Words: 0.018*\"council\" + 0.018*\"countri\" + 0.017*\"nation\" + 0.015*\"tasmanian\" + 0.015*\"hour\" + 0.014*\"labor\" + 0.013*\"plan\" + 0.012*\"say\" + 0.011*\"citi\" + 0.011*\"state\"\n",
            "Topic: 3 \n",
            "Words: 0.023*\"perth\" + 0.020*\"women\" + 0.018*\"miss\" + 0.018*\"sydney\" + 0.013*\"protest\" + 0.012*\"guilti\" + 0.011*\"search\" + 0.011*\"water\" + 0.011*\"john\" + 0.010*\"continu\"\n",
            "Topic: 4 \n",
            "Words: 0.028*\"elect\" + 0.018*\"say\" + 0.017*\"live\" + 0.017*\"famili\" + 0.016*\"chang\" + 0.011*\"park\" + 0.011*\"polit\" + 0.010*\"vote\" + 0.010*\"leader\" + 0.010*\"christma\"\n",
            "Topic: 5 \n",
            "Words: 0.036*\"trump\" + 0.028*\"queensland\" + 0.017*\"australia\" + 0.015*\"tasmania\" + 0.014*\"turnbul\" + 0.013*\"say\" + 0.012*\"student\" + 0.011*\"case\" + 0.010*\"million\" + 0.010*\"hobart\"\n",
            "Topic: 6 \n",
            "Words: 0.031*\"charg\" + 0.029*\"court\" + 0.019*\"face\" + 0.017*\"murder\" + 0.016*\"adelaid\" + 0.016*\"final\" + 0.015*\"woman\" + 0.015*\"accus\" + 0.015*\"donald\" + 0.015*\"open\"\n",
            "Topic: 7 \n",
            "Words: 0.046*\"polic\" + 0.030*\"year\" + 0.021*\"attack\" + 0.019*\"kill\" + 0.018*\"north\" + 0.017*\"crash\" + 0.017*\"south\" + 0.017*\"coast\" + 0.015*\"alleg\" + 0.012*\"melbourn\"\n",
            "Topic: 8 \n",
            "Words: 0.031*\"govern\" + 0.018*\"market\" + 0.013*\"australian\" + 0.013*\"share\" + 0.013*\"break\" + 0.012*\"warn\" + 0.011*\"busi\" + 0.011*\"high\" + 0.011*\"bank\" + 0.009*\"fund\"\n",
            "Topic: 9 \n",
            "Words: 0.016*\"health\" + 0.015*\"rural\" + 0.013*\"power\" + 0.012*\"servic\" + 0.012*\"price\" + 0.011*\"hous\" + 0.011*\"concern\" + 0.011*\"communiti\" + 0.010*\"royal\" + 0.009*\"farm\"\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l9EyW4PhKJ0L",
        "colab_type": "text"
      },
      "source": [
        "Cool! Can you distinguish different topics using the words in each topic and their corresponding weights?"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sko8PJASKJ0M",
        "colab_type": "text"
      },
      "source": [
        "### Running LDA using TF-IDF"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tLpHePRpKJ0M",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "lda_model_tfidf = gensim.models.LdaMulticore(corpus_tfidf, num_topics=10, id2word=dictionary, passes=2, workers=4)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U3VyHDR2KJ0O",
        "colab_type": "code",
        "outputId": "84398244-f77f-4bf7-899d-2ed0dc97257a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 197
        }
      },
      "source": [
        "for idx, topic in lda_model_tfidf.print_topics(-1):\n",
        "    print('Topic: {} Word: {}'.format(idx, topic))"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Topic: 0 Word: 0.011*\"crash\" + 0.011*\"kill\" + 0.009*\"donald\" + 0.008*\"dead\" + 0.007*\"attack\" + 0.007*\"north\" + 0.006*\"korea\" + 0.006*\"truck\" + 0.006*\"wednesday\" + 0.006*\"monday\"\n",
            "Topic: 1 Word: 0.018*\"rural\" + 0.008*\"grandstand\" + 0.008*\"sentenc\" + 0.008*\"plead\" + 0.006*\"guilti\" + 0.006*\"nation\" + 0.005*\"disabl\" + 0.005*\"track\" + 0.005*\"drink\" + 0.005*\"product\"\n",
            "Topic: 2 Word: 0.022*\"countri\" + 0.020*\"hour\" + 0.011*\"news\" + 0.011*\"podcast\" + 0.007*\"abbott\" + 0.007*\"live\" + 0.007*\"ash\" + 0.006*\"mental\" + 0.006*\"malcolm\" + 0.005*\"health\"\n",
            "Topic: 3 Word: 0.015*\"trump\" + 0.009*\"govern\" + 0.007*\"fund\" + 0.006*\"sport\" + 0.006*\"council\" + 0.006*\"plan\" + 0.005*\"water\" + 0.005*\"juli\" + 0.004*\"drought\" + 0.004*\"land\"\n",
            "Topic: 4 Word: 0.011*\"drum\" + 0.011*\"weather\" + 0.007*\"david\" + 0.007*\"climat\" + 0.006*\"march\" + 0.006*\"friday\" + 0.006*\"dairi\" + 0.006*\"australia\" + 0.005*\"john\" + 0.005*\"wild\"\n",
            "Topic: 5 Word: 0.010*\"final\" + 0.008*\"leagu\" + 0.008*\"world\" + 0.006*\"australia\" + 0.006*\"beat\" + 0.006*\"open\" + 0.006*\"tuesday\" + 0.005*\"decemb\" + 0.005*\"toni\" + 0.005*\"star\"\n",
            "Topic: 6 Word: 0.014*\"interview\" + 0.009*\"royal\" + 0.008*\"commiss\" + 0.007*\"liber\" + 0.007*\"elect\" + 0.007*\"michael\" + 0.006*\"stori\" + 0.006*\"septemb\" + 0.006*\"thursday\" + 0.006*\"senat\"\n",
            "Topic: 7 Word: 0.014*\"polic\" + 0.013*\"charg\" + 0.013*\"murder\" + 0.010*\"court\" + 0.009*\"death\" + 0.008*\"assault\" + 0.008*\"woman\" + 0.008*\"alleg\" + 0.006*\"jail\" + 0.006*\"arrest\"\n",
            "Topic: 8 Word: 0.008*\"queensland\" + 0.008*\"market\" + 0.006*\"share\" + 0.006*\"rat\" + 0.006*\"rise\" + 0.006*\"peter\" + 0.005*\"price\" + 0.005*\"octob\" + 0.005*\"asylum\" + 0.005*\"marriag\"\n",
            "Topic: 9 Word: 0.011*\"christma\" + 0.011*\"turnbul\" + 0.007*\"street\" + 0.007*\"novemb\" + 0.007*\"wall\" + 0.006*\"wrap\" + 0.005*\"island\" + 0.005*\"say\" + 0.005*\"indonesia\" + 0.005*\"busi\"\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7ORNmdclKJ0R",
        "colab_type": "text"
      },
      "source": [
        "### Classification of the topics"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0zCnGGXQKJ0S",
        "colab_type": "text"
      },
      "source": [
        "### Performance evaluation by classifying sample document using LDA Bag of Words model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G1-lDbWEKJ0T",
        "colab_type": "code",
        "outputId": "c2f1deaf-1476-4cf1-e3a7-535a1e1c7b48",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "processed_docs[4310]"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['rain', 'help', 'dampen', 'bushfir']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ICfNuzJUKJ0U",
        "colab_type": "code",
        "outputId": "8ed84d89-81d9-4ab3-e45f-49e9ec55af93",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 557
        }
      },
      "source": [
        "for index, score in sorted(lda_model[bow_corpus[4310]], key=lambda tup: -1*tup[1]):\n",
        "    print(\"\\nScore: {}\\t \\nTopic: {}\".format(score, lda_model.print_topic(index, 10)))"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Score: 0.3099364936351776\t \n",
            "Topic: 0.031*\"govern\" + 0.018*\"market\" + 0.013*\"australian\" + 0.013*\"share\" + 0.013*\"break\" + 0.012*\"warn\" + 0.011*\"busi\" + 0.011*\"high\" + 0.011*\"bank\" + 0.009*\"fund\"\n",
            "\n",
            "Score: 0.28417813777923584\t \n",
            "Topic: 0.016*\"health\" + 0.015*\"rural\" + 0.013*\"power\" + 0.012*\"servic\" + 0.012*\"price\" + 0.011*\"hous\" + 0.011*\"concern\" + 0.011*\"communiti\" + 0.010*\"royal\" + 0.009*\"farm\"\n",
            "\n",
            "Score: 0.2658781111240387\t \n",
            "Topic: 0.028*\"elect\" + 0.018*\"say\" + 0.017*\"live\" + 0.017*\"famili\" + 0.016*\"chang\" + 0.011*\"park\" + 0.011*\"polit\" + 0.010*\"vote\" + 0.010*\"leader\" + 0.010*\"christma\"\n",
            "\n",
            "Score: 0.020003201439976692\t \n",
            "Topic: 0.023*\"perth\" + 0.020*\"women\" + 0.018*\"miss\" + 0.018*\"sydney\" + 0.013*\"protest\" + 0.012*\"guilti\" + 0.011*\"search\" + 0.011*\"water\" + 0.011*\"john\" + 0.010*\"continu\"\n",
            "\n",
            "Score: 0.02000184915959835\t \n",
            "Topic: 0.023*\"death\" + 0.020*\"hospit\" + 0.016*\"trial\" + 0.014*\"driver\" + 0.012*\"street\" + 0.011*\"flood\" + 0.010*\"road\" + 0.010*\"babi\" + 0.010*\"east\" + 0.009*\"wall\"\n",
            "\n",
            "Score: 0.020000910386443138\t \n",
            "Topic: 0.046*\"polic\" + 0.030*\"year\" + 0.021*\"attack\" + 0.019*\"kill\" + 0.018*\"north\" + 0.017*\"crash\" + 0.017*\"south\" + 0.017*\"coast\" + 0.015*\"alleg\" + 0.012*\"melbourn\"\n",
            "\n",
            "Score: 0.020000774413347244\t \n",
            "Topic: 0.036*\"trump\" + 0.028*\"queensland\" + 0.017*\"australia\" + 0.015*\"tasmania\" + 0.014*\"turnbul\" + 0.013*\"say\" + 0.012*\"student\" + 0.011*\"case\" + 0.010*\"million\" + 0.010*\"hobart\"\n",
            "\n",
            "Score: 0.02000056393444538\t \n",
            "Topic: 0.018*\"council\" + 0.018*\"countri\" + 0.017*\"nation\" + 0.015*\"tasmanian\" + 0.015*\"hour\" + 0.014*\"labor\" + 0.013*\"plan\" + 0.012*\"say\" + 0.011*\"citi\" + 0.011*\"state\"\n",
            "\n",
            "Score: 0.019999999552965164\t \n",
            "Topic: 0.025*\"world\" + 0.019*\"test\" + 0.016*\"interview\" + 0.014*\"australia\" + 0.012*\"leagu\" + 0.010*\"close\" + 0.010*\"darwin\" + 0.009*\"take\" + 0.009*\"deal\" + 0.009*\"star\"\n",
            "\n",
            "Score: 0.019999999552965164\t \n",
            "Topic: 0.031*\"charg\" + 0.029*\"court\" + 0.019*\"face\" + 0.017*\"murder\" + 0.016*\"adelaid\" + 0.016*\"final\" + 0.015*\"woman\" + 0.015*\"accus\" + 0.015*\"donald\" + 0.015*\"open\"\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ef79aDrIKJ0W",
        "colab_type": "text"
      },
      "source": [
        "Our test document has the highest probability to be part of the topic on the top."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3AHlFJPoKJ0X",
        "colab_type": "text"
      },
      "source": [
        "### Performance evaluation by classifying sample document using LDA TF-IDF model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tV8zlXkyKJ0X",
        "colab_type": "code",
        "outputId": "86455ffb-936e-43bc-94d9-1ed5ca35c5a1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 557
        }
      },
      "source": [
        "for index, score in sorted(lda_model_tfidf[bow_corpus[4310]], key=lambda tup: -1*tup[1]):\n",
        "    print(\"\\nScore: {}\\t \\nTopic: {}\".format(score, lda_model_tfidf.print_topic(index, 10)))"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Score: 0.5878947973251343\t \n",
            "Topic: 0.015*\"trump\" + 0.009*\"govern\" + 0.007*\"fund\" + 0.006*\"sport\" + 0.006*\"council\" + 0.006*\"plan\" + 0.005*\"water\" + 0.005*\"juli\" + 0.004*\"drought\" + 0.004*\"land\"\n",
            "\n",
            "Score: 0.25208669900894165\t \n",
            "Topic: 0.010*\"final\" + 0.008*\"leagu\" + 0.008*\"world\" + 0.006*\"australia\" + 0.006*\"beat\" + 0.006*\"open\" + 0.006*\"tuesday\" + 0.005*\"decemb\" + 0.005*\"toni\" + 0.005*\"star\"\n",
            "\n",
            "Score: 0.020004238933324814\t \n",
            "Topic: 0.014*\"polic\" + 0.013*\"charg\" + 0.013*\"murder\" + 0.010*\"court\" + 0.009*\"death\" + 0.008*\"assault\" + 0.008*\"woman\" + 0.008*\"alleg\" + 0.006*\"jail\" + 0.006*\"arrest\"\n",
            "\n",
            "Score: 0.020002644509077072\t \n",
            "Topic: 0.014*\"interview\" + 0.009*\"royal\" + 0.008*\"commiss\" + 0.007*\"liber\" + 0.007*\"elect\" + 0.007*\"michael\" + 0.006*\"stori\" + 0.006*\"septemb\" + 0.006*\"thursday\" + 0.006*\"senat\"\n",
            "\n",
            "Score: 0.02000254951417446\t \n",
            "Topic: 0.018*\"rural\" + 0.008*\"grandstand\" + 0.008*\"sentenc\" + 0.008*\"plead\" + 0.006*\"guilti\" + 0.006*\"nation\" + 0.005*\"disabl\" + 0.005*\"track\" + 0.005*\"drink\" + 0.005*\"product\"\n",
            "\n",
            "Score: 0.02000238560140133\t \n",
            "Topic: 0.011*\"drum\" + 0.011*\"weather\" + 0.007*\"david\" + 0.007*\"climat\" + 0.006*\"march\" + 0.006*\"friday\" + 0.006*\"dairi\" + 0.006*\"australia\" + 0.005*\"john\" + 0.005*\"wild\"\n",
            "\n",
            "Score: 0.020002208650112152\t \n",
            "Topic: 0.008*\"queensland\" + 0.008*\"market\" + 0.006*\"share\" + 0.006*\"rat\" + 0.006*\"rise\" + 0.006*\"peter\" + 0.005*\"price\" + 0.005*\"octob\" + 0.005*\"asylum\" + 0.005*\"marriag\"\n",
            "\n",
            "Score: 0.02000194415450096\t \n",
            "Topic: 0.011*\"christma\" + 0.011*\"turnbul\" + 0.007*\"street\" + 0.007*\"novemb\" + 0.007*\"wall\" + 0.006*\"wrap\" + 0.005*\"island\" + 0.005*\"say\" + 0.005*\"indonesia\" + 0.005*\"busi\"\n",
            "\n",
            "Score: 0.020001348108053207\t \n",
            "Topic: 0.011*\"crash\" + 0.011*\"kill\" + 0.009*\"donald\" + 0.008*\"dead\" + 0.007*\"attack\" + 0.007*\"north\" + 0.006*\"korea\" + 0.006*\"truck\" + 0.006*\"wednesday\" + 0.006*\"monday\"\n",
            "\n",
            "Score: 0.020001176744699478\t \n",
            "Topic: 0.022*\"countri\" + 0.020*\"hour\" + 0.011*\"news\" + 0.011*\"podcast\" + 0.007*\"abbott\" + 0.007*\"live\" + 0.007*\"ash\" + 0.006*\"mental\" + 0.006*\"malcolm\" + 0.005*\"health\"\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "40bcjetvKJ0a",
        "colab_type": "text"
      },
      "source": [
        "Our test document has the highest probability to be part of the topic on the top."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "h0Om5cAnKJ0b",
        "colab_type": "text"
      },
      "source": [
        "### Testing model on unseen document"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u2UXHhoBKJ0b",
        "colab_type": "code",
        "outputId": "3535116b-01ab-4c1e-b4c8-7f90265cf803",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 197
        }
      },
      "source": [
        "unseen_document = 'How a Pentagon deal became an identity crisis for Google'\n",
        "bow_vector = dictionary.doc2bow(preprocess(unseen_document))\n",
        "\n",
        "for index, score in sorted(lda_model[bow_vector], key=lambda tup: -1*tup[1]):\n",
        "    print(\"Score: {}\\t Topic: {}\".format(score, lda_model.print_topic(index, 5)))"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Score: 0.3499976098537445\t Topic: 0.028*\"elect\" + 0.018*\"say\" + 0.017*\"live\" + 0.017*\"famili\" + 0.016*\"chang\"\n",
            "Score: 0.18333333730697632\t Topic: 0.031*\"govern\" + 0.018*\"market\" + 0.013*\"australian\" + 0.013*\"share\" + 0.013*\"break\"\n",
            "Score: 0.18333320319652557\t Topic: 0.036*\"trump\" + 0.028*\"queensland\" + 0.017*\"australia\" + 0.015*\"tasmania\" + 0.014*\"turnbul\"\n",
            "Score: 0.1833295226097107\t Topic: 0.025*\"world\" + 0.019*\"test\" + 0.016*\"interview\" + 0.014*\"australia\" + 0.012*\"leagu\"\n",
            "Score: 0.016670528799295425\t Topic: 0.023*\"death\" + 0.020*\"hospit\" + 0.016*\"trial\" + 0.014*\"driver\" + 0.012*\"street\"\n",
            "Score: 0.01666843704879284\t Topic: 0.016*\"health\" + 0.015*\"rural\" + 0.013*\"power\" + 0.012*\"servic\" + 0.012*\"price\"\n",
            "Score: 0.016667399555444717\t Topic: 0.023*\"perth\" + 0.020*\"women\" + 0.018*\"miss\" + 0.018*\"sydney\" + 0.013*\"protest\"\n",
            "Score: 0.01666666753590107\t Topic: 0.018*\"council\" + 0.018*\"countri\" + 0.017*\"nation\" + 0.015*\"tasmanian\" + 0.015*\"hour\"\n",
            "Score: 0.01666666753590107\t Topic: 0.031*\"charg\" + 0.029*\"court\" + 0.019*\"face\" + 0.017*\"murder\" + 0.016*\"adelaid\"\n",
            "Score: 0.01666666753590107\t Topic: 0.046*\"polic\" + 0.030*\"year\" + 0.021*\"attack\" + 0.019*\"kill\" + 0.018*\"north\"\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ggG3g9usNiGL",
        "colab_type": "text"
      },
      "source": [
        "**Reference**:  \n",
        "https://towardsdatascience.com/topic-modeling-and-latent-dirichlet-allocation-in-python-9bf156893c24\n",
        "https://medium.com/analytics-vidhya/topic-modeling-using-lda-and-gibbs-sampling-explained-49d49b3d1045\n",
        "https://medium.com/@lettier/how-does-lda-work-ill-explain-using-emoji-108abf40fa7d"
      ]
    }
  ]
}